{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.尝试不同的权重初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "\n",
    "\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "\n",
    "from keras import backend as K\n",
    "import time\n",
    "from keras import initializers\n",
    "from keras.objectives import categorical_crossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Model(kernel_init,bias_init):\n",
    "    with tf.name_scope('reshape'):\n",
    "        x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "    net = Conv2D(32, kernel_size=[5,5], strides=[1,1],activation='relu', \n",
    "                     padding='same',\n",
    "                     kernel_initializer = kernel_init, #卷积权重初始化\n",
    "                     bias_initializer = bias_init, #偏置权重初始化\n",
    "                    input_shape=[28,28,1])(x_image)\n",
    "    net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "    net = Conv2D(64, kernel_size=[5,5], strides=[1,1],activation='relu',\n",
    "                     kernel_initializer = kernel_init,\n",
    "                     bias_initializer = bias_init,\n",
    "                    padding='same')(net)\n",
    "    net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "    net = Flatten()(net)\n",
    "    net = Dense(1000, activation='relu')(net) \n",
    "    net = Dense(10,activation='softmax')(net)\n",
    "    \n",
    "    cross_entropy = tf.reduce_mean(categorical_crossentropy(y_, net))\n",
    "    l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "\n",
    "    total_loss = cross_entropy + 7e-5*l2_loss\n",
    "\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "    sess = tf.Session()\n",
    "    K.set_session(sess)\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    start = time.time()\n",
    "    for step in range(3000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        lr = 0.01\n",
    "        _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "                   [train_step, cross_entropy, l2_loss, total_loss], \n",
    "                   feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr})\n",
    "\n",
    "        if (step+1) % 100 == 0:\n",
    "            #print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "             #  (step+1, loss, l2_loss_value, total_loss_value))\n",
    "            #Test trained model\n",
    "            correct_prediction = tf.equal(tf.argmax(net, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(\"训练集准确度为\",sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "        if (step+1) % 1000 == 0:\n",
    "            print(\"测试集准确度为\",sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                        y_: mnist.test.labels}))\n",
    "            end = time.time()\n",
    "            print(\"%d步所用的时间为%fs\"%(step+1,end-start))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化采取<keras.initializers.TruncatedNormal object at 0x000002DB632E9B70>时:\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9606\n",
      "1000步所用的时间为312.894830s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9725\n",
      "2000步所用的时间为624.974554s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9808\n",
      "3000步所用的时间为935.919421s\n",
      "初始化采取<keras.initializers.RandomUniform object at 0x000002DB632E9CF8>时:\n",
      "训练集准确度为 0.83\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.96\n",
      "测试集准确度为 0.9497\n",
      "1000步所用的时间为318.235318s\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9683\n",
      "2000步所用的时间为635.037993s\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9768\n",
      "3000步所用的时间为952.216479s\n",
      "初始化采取<keras.initializers.RandomNormal object at 0x000002DB632E9BA8>时:\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "测试集准确度为 0.9639\n",
      "1000步所用的时间为328.497020s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9746\n",
      "2000步所用的时间为654.673563s\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9798\n",
      "3000步所用的时间为979.673112s\n",
      "初始化采取<keras.initializers.Ones object at 0x000002DB632E9D68>时:\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.06\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.07\n",
      "训练集准确度为 0.06\n",
      "训练集准确度为 0.12\n",
      "测试集准确度为 0.098\n",
      "1000步所用的时间为333.415127s\n",
      "训练集准确度为 0.17\n",
      "训练集准确度为 0.06\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.11\n",
      "测试集准确度为 0.098\n",
      "2000步所用的时间为665.348374s\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.07\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.12\n",
      "测试集准确度为 0.098\n",
      "3000步所用的时间为1000.627138s\n",
      "初始化采取<keras.initializers.Zeros object at 0x000002DB632E9A90>时:\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.04\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.16\n",
      "测试集准确度为 0.1135\n",
      "1000步所用的时间为344.828354s\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.07\n",
      "测试集准确度为 0.1135\n",
      "2000步所用的时间为688.791106s\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.05\n",
      "训练集准确度为 0.1\n",
      "测试集准确度为 0.1135\n",
      "3000步所用的时间为1032.355630s\n"
     ]
    }
   ],
   "source": [
    "#尝试常用的初始化分布\n",
    "#1.调试不同初始化相应参数\n",
    "#2.比较不同初始化训练结果，\n",
    "\n",
    "zeros = initializers.Zeros()#全零初始化\n",
    "ones = initializers.Ones() #全一初始化\n",
    "rn = initializers.RandomNormal(mean = 0.0,stddev = 0.1) #正态分布初始化\n",
    "ru = initializers.RandomUniform(minval=-0.1, maxval=0.1) #均匀分布初始化\n",
    "tn = initializers.TruncatedNormal(mean=0.0, stddev=0.1) #截断高斯分布初始化\n",
    "init = [tn,ru,rn,ones,zeros]\n",
    "bias_init = zeros #偏置一般使用全零初始化\n",
    "for kernel_init in init:\n",
    "    print(\"初始化采取%s时:\"%kernel_init)\n",
    "    Model(kernel_init,bias_init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述优化可以看出，使用截断高斯分布对核权重进行初始化可以明显改善默认参数下的准确度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
